#!/usr/bin/env python
# -*- coding:utf-8 -*-
# @FileName  :get_book_list.py
# @Time      :2023/10/16 
# @Author    :CL
# @email     :1037654919@qq.com

import requests
from utils import mongo_manager,get_proxy
from retrying import retry
@retry(stop_max_attempt_number = 3)
def get_book_list(    params = {
        "page": "1",
        "category_id": "13200",
        "sub_category_id": "13201"
    }):
    headers = {
        "Accept": "application/json, text/javascript, */*; q=0.01",
        "Accept-Language": "zh-CN,zh;q=0.9",
        "Cache-Control": "no-cache",
        "Connection": "keep-alive",
        "Pragma": "no-cache",
        "Referer": "https://jinghecx.com/responsive/list?subCategoryId=13201",
        "Sec-Fetch-Dest": "empty",
        "Sec-Fetch-Mode": "cors",
        "Sec-Fetch-Site": "same-origin",
        "User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36",
        "X-Requested-With": "XMLHttpRequest",
        "responsive-org": "clcn",
        "responsive-session-id": "",
        "sec-ch-ua": "\"Not.A/Brand\";v=\"8\", \"Chromium\";v=\"114\", \"Google Chrome\";v=\"114\"",
        "sec-ch-ua-mobile": "?0",
        "sec-ch-ua-platform": "\"Linux\""
    }
    cookies={}
    url = "https://jinghecx.com/api/responsive/book-list"
    # params = {
    #     "page": "1",
    #     "category_id": "13200",
    #     "sub_category_id": "13201"
    # }
    response = requests.get(url, headers=headers,proxies=get_proxy(),timeout=10, params=params)

    # print(response.text)
    # print(response)
    return  response.json()
def pool_run(seed):
    print(seed)
    qq_book_list = mongo_manager('qq_book_list')
    qq_sub_category_list = mongo_manager('qq_sub_category_list')

    category_id = seed['father_id']
    sub_category_id = seed['sub_category_id']
    params = {
        "page":"1",
        "category_id":category_id,
        "sub_category_id": sub_category_id
    }
    book_list = get_book_list(params=params)
    datas = book_list['data']['book_list']
    # maxpage = book_list['data']['max_page']
    # seed['maxpage'] = maxpage
    # try:
    #     qq_sub_category_list.insertOne(seed)
    # except Exception as e:
    #     qq_sub_category_list.updateOne({"_id": seed['_id']}, seed)
    for data in datas:
        data['_id'] = data["book_id"]
        data['sub_category_id'] = sub_category_id
        data['sub_category_name'] = seed['sub_category_name']
        try:
            qq_book_list.insertOne(data)
        except Exception as e:
            qq_book_list.updateOne({"_id": data['_id']}, data)
    maxpage = book_list['data']['max_page']
    print(seed['father_name'],seed['sub_category_name'],'maxpage:',maxpage)
    begin =2
    # 临时，可删除 begin默认为2
    if seed['sub_category_name'] == '经史子集':
        begin = 369
    if seed['sub_category_name'] == '一般管理学':
        begin = 70
    if seed['sub_category_name'] == '中国现当代随笔':
        begin = 44
    if seed['sub_category_name'] == '外国经典':
        begin = 44
    if seed['sub_category_name'] == '市场/营销':
        begin = 33
    if seed['sub_category_name'] == '成功/激励':
        begin = 45
    if seed['sub_category_name'] == '中医':
        begin = 27
    if seed['sub_category_name'] == '其他':
        begin = 20
    if seed['sub_category_name'] == '专用软件':
        begin = 27
    for page in range(begin, maxpage + 1):
        params = {
            "page": page,
            "category_id":category_id,
            "sub_category_id": sub_category_id
        }
        book_list = get_book_list(params=params)
        datas = book_list['data']['book_list']
        for data in datas:
            data['_id'] = data["book_id"]
            data['sub_category_id'] =sub_category_id
            data['sub_category_name'] =seed['sub_category_name']
            try:
                qq_book_list.insertOne(data)
            except Exception as e:
                qq_book_list.updateOne({"_id": data['_id']}, data)
    seed['status'] = 1
    qq_sub_category_list.updateOne({"_id": seed['_id']}, seed)
    qq_sub_category_list.close()
    qq_book_list.close()


if __name__ == "__main__":
    print()
    from multiprocessing import Pool

    qq_sub_category_list = mongo_manager('qq_sub_category_list')
    seeds = qq_sub_category_list.findAll({'status':None})  #{'status':None,'maxpage':{"$lt":40}}
    lists =[]
    for seed in seeds:
        lists.append(seed)
    print('-' * 20, len(lists))
    pool = Pool(processes=min(30, len(lists) + 1))
    pool.map(pool_run, lists)
    pool.close()  # 关闭进程池，不再接受新的进程
    pool.join()
    qq_sub_category_list.close()